package com.terren.spider.run.basic;


import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CompletableFuture;

import org.apache.log4j.Logger;

import com.terren.spider.db.service.EntryService;
import com.terren.spider.db.service.SpiderLogService;
import com.terren.spider.entity.biz.AppData;
import com.terren.spider.entity.biz.Article;
import com.terren.spider.entity.biz.ArticleReply;
import com.terren.spider.entity.biz.ArticleSearch;
import com.terren.spider.entity.biz.ArticleTopic;
import com.terren.spider.entity.biz.Chengjiaofang;
import com.terren.spider.entity.biz.DetailXhs;
import com.terren.spider.entity.biz.Douyin;
import com.terren.spider.entity.biz.ElectronicCommerce;
import com.terren.spider.entity.biz.Ershoufang;
import com.terren.spider.entity.biz.TiebaUser;
import com.terren.spider.entity.biz.UnionComments;
import com.terren.spider.entity.biz.UnionUserInfo;
import com.terren.spider.entity.biz.XhsUserDetail;
import com.terren.spider.entity.biz.ZhihuContent;
import com.terren.spider.entity.biz.ZhihuUser;
import com.terren.spider.entity.common.SpiderLog;
import com.terren.spider.entity.core.CatagoryType;
import com.terren.spider.entity.core.Entry;
import com.terren.spider.handler.impl.HtmlSpiderHandlerImpl;
import com.terren.spider.util.common.CommonUtil;
import com.terren.spider.util.common.LogUtil;

public abstract class BasicSpiderRunner<T> {
	protected  EntryService entryService;
	protected  HtmlSpiderHandlerImpl<T> handler;
	protected Logger log;
	protected SpiderLogService spiderLogService;// 爬虫日志service
	private List<CompletableFuture<Void>> list=new ArrayList<>();
	private Integer excuteNum=2; //每次执行的数量
	public BasicSpiderRunner() {
		super();
		initRunner();
	}
	
	public abstract List<Entry> getEntrys();


	/**
	 * 根据catagory创建spider handler
	 * @param entry
	 * @return
	 * @throws Exception
	 */
	@SuppressWarnings("unchecked")
	public HtmlSpiderHandlerImpl<T> spiderHandler(Entry entry) throws Exception {
		CatagoryType catagoryType = CatagoryType.valueOfType(entry.getCatagoryId());
		if (null != catagoryType) {
			switch (catagoryType) {
			case ARTICLE_SEARCH:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) ArticleSearch.class);
			case TIEBA:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) ArticleSearch.class);
			case TIEBA_USER:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) TiebaUser.class);
			case ERSHOUFANG:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) Ershoufang.class);
			case CHENGJIAOFANG:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) Chengjiaofang.class);
			case ELECTRONIC_COMMERCE:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) ElectronicCommerce.class);
			case APP_DATA:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) AppData.class);
			case XIAOHONGSHU:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) DetailXhs.class);
			case ARTICLE_TOPIC:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) ArticleTopic.class);
			case ARTICLE_REPLY:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) ArticleReply.class);
			case ZHIHU_USER:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) ZhihuUser.class);
			case XIAOHONGSHU_USER:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) XhsUserDetail.class);
			case THEPAPER_COMMENTS:	
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) UnionComments.class);
			case THEPAPER_COMMENTS_USER:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) UnionUserInfo.class);
			case ZHIHU_CONTENT:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) ZhihuContent.class);//`t_regulartype` 里categirud=1
			case DOUYIN_KEYWOED:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) Douyin.class);
			default:
				return new HtmlSpiderHandlerImpl<T>(entry, (Class<T>) Article.class);
			}
		}
		return null;
	}


	public  void postInitEntry(Entry entry)throws Exception{

	}
	public  void postRun(Entry entry) {

	}


	protected void initRunner() {
		LogUtil.initLogger();// 初始化log4j
		log = LogUtil.logger;
		entryService = CommonUtil.getBean(EntryService.class);
		spiderLogService = CommonUtil.getBean(SpiderLogService.class);
	}

	public void run() throws Exception {
		long startTime, endTime;
		startTime = System.currentTimeMillis();
		log.info("【爬虫开始】请耐心等待...");
		ArrayBlockingQueue<Entry> queue =null;
		List<Entry> entries = getEntrys();
		if(entries.size()>0){
			queue= new ArrayBlockingQueue<Entry>(entries.size());

			for (Entry entry : entries) {
				queue.add(entry);
			}

			while(queue.size()>0){

				Entry entry = queue.take();
				CompletableFuture<Void> async = CompletableFuture.runAsync( ()-> {	
					SpiderLog slog = new SpiderLog();
					String message = entry.getEntryDesc()+"爬取成功";
					Integer status = 1;//默认成功
					Integer size = 0;
					try{
						//记录爬虫日志
						slog = saveLog(entry);
						postInitEntry(entry);
						handler = spiderHandler(entry);//根据catagory创建spider handler
						String[]urls = handler.findUrl();
						size = handler.parsePage(urls);
					}catch(Exception e){
						message = entry.getEntryDesc()+"爬取失败     "+e.getMessage();
						status = 0;
						//throw new IllegalArgumentException(e.getMessage());
					}finally{
						postRun(entry);
						//更新爬虫日志
						try {
							updateLog(slog,entry,message,size,status);
						} catch (Exception e) {
							e.printStackTrace();
						}
					}

				});
				list.add(async);

				if(list.size()==excuteNum) {


					@SuppressWarnings("rawtypes")
					CompletableFuture[] array = list.toArray(new CompletableFuture[list.size()]);

					CompletableFuture<Void> result=CompletableFuture.allOf(array);
					result.join();	
					list.clear();
				}

			}		
			//防止list中有未执行的
			if(list.size()>0) {

				@SuppressWarnings("rawtypes")
				CompletableFuture[] array = list.toArray(new CompletableFuture[list.size()]);

				CompletableFuture<Void> result=CompletableFuture.allOf(array);
				result.join();	
				list.clear();
			}
		}
		endTime = System.currentTimeMillis();
		log.info("【爬虫结束】,共耗时约" + ((endTime - startTime) / 1000 / 60) + "分。");
	}

	//初始化日志
	protected SpiderLog saveLog(Entry entry) throws Exception{
		SpiderLog slog = new SpiderLog();
		slog.setEntryId(entry.getEntryId());
		slog.setBeginTime(new Date());
		slog.setStatus(0);//默认状态是爬取失败
		slog.setTaskid(entry.getTaskId());
		spiderLogService.addSpiderLog(slog);
		return slog;
	}

	//更新日志状态
	protected void updateLog(SpiderLog slog,Entry entry,String message,Integer size,Integer status) throws Exception{
		slog.setMessage(message);
		slog.setLinks(size);
		slog.setEntryId(entry.getEntryId());
		slog.setEndTime(new Date());
		slog.setStatus(status);
		spiderLogService.updateSpiderLog(slog);
	}
}
